package cn.edu.suda.ada.fetcher;

import java.io.IOException;
import java.net.URISyntaxException;
import java.util.logging.Logger;

import org.apache.http.HttpEntity;
import org.apache.http.HttpHost;
import org.apache.http.HttpResponse;
import org.apache.http.HttpStatus;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.HttpClient;
import org.apache.http.client.config.CookieSpecs;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.params.ClientPNames;
import org.apache.http.client.params.CookiePolicy;
import org.apache.http.client.params.HttpClientParams;
import org.apache.http.config.Registry;
import org.apache.http.config.RegistryBuilder;
import org.apache.http.conn.params.ConnRouteParams;
import org.apache.http.cookie.Cookie;
import org.apache.http.cookie.CookieOrigin;
import org.apache.http.cookie.CookieSpec;
import org.apache.http.cookie.CookieSpecFactory;
import org.apache.http.cookie.CookieSpecProvider;
import org.apache.http.cookie.MalformedCookieException;
import org.apache.http.entity.mime.Header;
import org.apache.http.impl.client.AbstractHttpClient;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.conn.DefaultHttpRoutePlanner;
import org.apache.http.impl.conn.DefaultProxyRoutePlanner;
import org.apache.http.impl.cookie.BestMatchSpecFactory;
import org.apache.http.impl.cookie.BrowserCompatSpec;
import org.apache.http.impl.cookie.BrowserCompatSpecFactory;
import org.apache.http.params.BasicHttpParams;
import org.apache.http.params.HttpConnectionParams;
import org.apache.http.params.HttpParams;
import org.apache.http.protocol.HttpContext;
import org.apache.http.util.EntityUtils;

import com.mysql.jdbc.StringUtils;

import cn.edu.suda.ada.model.FetchedPage;
import cn.edu.suda.ada.model.Proxy;
import cn.edu.suda.ada.queue.StaticUrlQueue;

public class PageFetcher {
	private static final Logger Log = Logger.getLogger(PageFetcher.class.getName());
	CloseableHttpClient  client;
	

	
	/**
	 * 创建HttpClient实例，并初始化连接参数
	 */
	public PageFetcher(){
		// 设置超时时间
//		HttpParams params = new BasicHttpParams();
//	    HttpConnectionParams.setConnectionTimeout(params, 80 * 1000);
//	    HttpConnectionParams.setSoTimeout(params, 80 * 1000);	    
		client = new DefaultHttpClient();//DefaultHttpClient(params)
		
//		((AbstractHttpClient) client).getCookieSpecs().register("easy", csf);
//		client.getParams().setParameter(ClientPNames.COOKIE_POLICY, "easy");
	}
	
	/**
	 * 主动关闭HttpClient连接
	 */
	public void close(){
		try {
			client.close();
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		};
	}
	
	/**
	 * 根据url爬取网页内容
	 * @param url
	 * @return
	 */
	public FetchedPage getContentFromUrl(String url){
		String content = null;
		int statusCode = 500;
		
		// 创建Get请求，并设置Header
		HttpGet getHttp = new HttpGet(url);	
		getHttp.setHeader("User-Agent","Mozilla/5.0 (Windows NT 6.1; WOW64; rv:33.0) Gecko/20100101 Firefox/33.0");
//		getHttp.getParams().setParameter(ClientPNames.COOKIE_POLICY, CookiePolicy.BROWSER_COMPATIBILITY);
//		((AbstractHttpClient) client).getCookieSpecs().register("easy", csf);
//		client.getParams().setParameter(ClientPNames.COOKIE_POLICY, CookiePolicy.BROWSER_COMPATIBILITY);
		HttpResponse response;
		
		try{
			// 获得信息载体
			response = client.execute(getHttp);
			statusCode = response.getStatusLine().getStatusCode();
			HttpEntity entity = response.getEntity();	
			
			if(entity != null){
				// 转化为文本信息, 设置爬取网页的字符集，防止乱码
				content = EntityUtils.toString(entity, "UTF-8");
				System.out.println(content);
			}
		}
		catch(Exception e){
			e.printStackTrace();
			
			// 因请求超时等问题产生的异常，将URL放回待抓取队列，重新爬取
			Log.info(">> Put back url: " + url);
			StaticUrlQueue.addFirstElement(url);
		}

		return new FetchedPage(url, content, statusCode);
	}
	/**
	 * 设置header测试方法
	 * @param 
	 * @return
	 */
	public void header() {
		HttpClient httpClient = new DefaultHttpClient();
		try {
			HttpGet httpget = new HttpGet("http://bj.ganji.com/fang1/");
//			httpget.setHeader("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
//			httpget.setHeader("Accept-Encoding", "gzip, deflate");
//			httpget.setHeader("Accept-Language", "zh-cn,zh;q=0.8,en-us;q=0.5,en;q=0.3");
//			httpget.setHeader("Connection", "keep-alive");
			httpget.setHeader("User-Agent", "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:33.0) Gecko/20100101 Firefox/33.0");
						
			HttpResponse response = httpClient.execute(httpget);
			HttpEntity entity = response.getEntity();
			System.out.println(response.getStatusLine()); //状态码
			if(entity != null) {
				String content = EntityUtils.toString(entity, "UTF-8");
				System.out.println(content);
//				System.out.println(entity.getContentLength());
//				System.out.println(entity.getContent());
			}
		} catch (Exception e) {
			e.printStackTrace();
		}
	}
	
	/** 默认方法 
	 * 
	 * @method
	 */
	public FetchedPage getHTML(String url) throws ClientProtocolException, IOException {
		int statusCode=500;
		String context="";
		
//		String[] html = new String[2];
//		html[1] = "null";
		RequestConfig requestConfig = RequestConfig.custom()
				.setSocketTimeout(80000)   //socket超时
				.setConnectTimeout(80000)   //connect超时
				.build();
		CloseableHttpClient httpClient = HttpClients.custom()
				.setDefaultRequestConfig(requestConfig)
				.build();
		
		HttpGet httpGet = new HttpGet(url);
		httpGet.setHeader("User-Agent", "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:33.0) Gecko/20100101 Firefox/33.0");
		httpGet.setHeader("Host","bj.ganji.com");
		
		try {
			CloseableHttpResponse response = httpClient.execute(httpGet);			
			statusCode = response.getStatusLine().getStatusCode();
			HttpEntity entity = response.getEntity();
			if(null!=entity){
				context=EntityUtils.toString(entity, "utf-8");
//				System.out.println(statusCode);
//				System.out.println(context);
			}
			//System.out.println(html);
		} catch (IOException e) {
			System.out.println("----------Connection timeout--------");
		}
		return new FetchedPage(url, context, statusCode);
	}
	/**
	 * 
	 * @method
	 */
	public FetchedPage getHTMLByCookie(String url, String hostName, int port) throws URISyntaxException, ClientProtocolException, IOException {
		//采用用户自定义的cookie策略
		HttpHost proxy = new HttpHost(hostName, port);
		DefaultProxyRoutePlanner routePlanner = new DefaultProxyRoutePlanner(proxy);
		CookieSpecProvider cookieSpecProvider = new CookieSpecProvider() {
			public CookieSpec create(HttpContext context) {
				return new BrowserCompatSpec() {
					@Override
					public void validate(Cookie cookie, CookieOrigin origin) throws MalformedCookieException {
						//Oh, I am easy...
					}
				};
			}
		};
		
		Registry<CookieSpecProvider> r = RegistryBuilder
				.<CookieSpecProvider> create()
				.register(CookieSpecs.BEST_MATCH, new BestMatchSpecFactory())
				.register(CookieSpecs.BROWSER_COMPATIBILITY, new BrowserCompatSpecFactory())
				.register("easy", cookieSpecProvider)
				.build();
		
		RequestConfig requestConfig = RequestConfig.custom()
				.setCookieSpec("easy")
				.setSocketTimeout(5000) //socket超时
				.setConnectTimeout(5000) //connect超时
				.build();
		CloseableHttpClient httpClient = HttpClients.custom()
				.setDefaultCookieSpecRegistry(r)
				.setRoutePlanner(routePlanner)
				.build();
		
		HttpGet httpGet = new HttpGet(url);
		httpGet.setConfig(requestConfig);
		httpGet.setHeader("User-Agent", "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:33.0) Gecko/20100101 Firefox/33.0");
		httpGet.setHeader("Host","bj.ganji.com");
		
		String html = "null"; //用于验证是否正常取到html
		int statusCode=500;
		try {
			CloseableHttpResponse response = httpClient.execute(httpGet);
			statusCode = response.getStatusLine().getStatusCode();
			html = EntityUtils.toString(response.getEntity(), "utf-8");	
			System.out.println(statusCode);
			System.out.println(html);
		} catch (IOException e) {
			System.out.println("----Connection timeout----");
		}
		return new FetchedPage(url, html, statusCode);
	}
	/**
	 * 通过代理抓页面
	 * @method
	 */
	public FetchedPage getHTMLbyProxy(String Url, String hostName, int port,String host) throws ClientProtocolException, IOException {
		System.out.println("正在爬"+Url);
//		System.out.println(hostName+":"+port+":"+host);
		
		int statusCode=500;
		String context ="";
		
		HttpHost proxy = new HttpHost(hostName, port);
		
		
		DefaultProxyRoutePlanner routePlanner = new DefaultProxyRoutePlanner(proxy);
		RequestConfig requestConfig = RequestConfig.custom()
				.setSocketTimeout(80*1000) //socket超时
				.setConnectTimeout(80*1000)//connect超时
//				.setRedirectsEnabled(false)//取消自动重定向
				.build();
		
				
		CloseableHttpClient httpClient = HttpClients.custom()
				.setRoutePlanner(routePlanner)
				.setDefaultRequestConfig(requestConfig)
				.build();
		
		HttpGet httpGet = new HttpGet(Url);
		httpGet.setHeader("User-Agent", "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:33.0) Gecko/20100101 Firefox/33.0");

		
		
		try {
			CloseableHttpResponse response = httpClient.execute(httpGet);
			statusCode = response.getStatusLine().getStatusCode();
			
//			System.out.println(statusCode);
			
			
			if(statusCode == HttpStatus.SC_OK) { //状态码200: OK
				context = EntityUtils.toString(response.getEntity(), "utf-8");
//				System.out.println("url："+Url);
//				System.out.println(statusCode);
//				System.out.println(context);
			}
			else {
//				System.out.println("！！！"+statusCode);
			}
			response.close();

		} catch (IOException e) {
			System.out.println("----Connection timeout----");
			Log.info(">> Put back url: " + Url);
		}
		finally{
			httpClient.close();
		}
		if(Url.contains("click.ganji.com")){
			Url=getRedirectUrl(Url, hostName, port, host);
		}
		if(Url.contains("jump.zhineng.58.com")){
			Url=getRedirectUrl(Url, hostName, port, host);
		}
		return new FetchedPage(Url, context, statusCode);
	}
	
	/**
	 * 获取重定向后的URL
	 * @throws IOException 
	 * @method
	 */
	public String getRedirectUrl(String Url, String hostName, int port,String host) throws IOException{
		
		int statusCode=500;
		
		HttpHost proxy = new HttpHost(hostName, port);
		
		
		DefaultProxyRoutePlanner routePlanner = new DefaultProxyRoutePlanner(proxy);
		RequestConfig requestConfig = RequestConfig.custom()
				.setSocketTimeout(80*1000) //socket超时
				.setConnectTimeout(80*1000)//connect超时
				.setRedirectsEnabled(false)//取消自动重定向
				.build();
				
		CloseableHttpClient httpClient = HttpClients.custom()
				.setRoutePlanner(routePlanner)
				.setDefaultRequestConfig(requestConfig)
				.build();
		
		HttpGet httpGet = new HttpGet(Url);
		httpGet.setHeader("User-Agent", "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:33.0) Gecko/20100101 Firefox/33.0");

		
		try {
			CloseableHttpResponse response = httpClient.execute(httpGet);
			statusCode = response.getStatusLine().getStatusCode();
			if(statusCode==302){
				org.apache.http.Header h=response.getFirstHeader("Location");
				Url=h.getValue();
//				System.out.println("重定向："+Url);
			}			
//			System.out.println(statusCode);

			response.close();
			//System.out.println(html); //打印返回的html
		} catch (IOException e) {
			System.out.println("----Connection timeout----");
			Log.info(">> Put back url: " + Url);
		}
		finally{
			httpClient.close();
		}
		return Url;
	}
	
//	public boolean isUsable(String url) throws ClientProtocolException, IOException{
//		for(int i=0;i<proxyIP.length;i++){
//			ip=proxyIP[i];
//			port=proxyPort[i];
//			FetchedPage fetchedPage=getHTMLbyProxy(url,ip, port);
//			if("".equals(fetchedPage.getContent())||null==fetchedPage.getContent()||fetchedPage.getStatusCode()==403){
//				continue;
//			}
//			else {
//				return true;
//			}
//		}
//		return false;
//	}
	
//	/**
//	 * 设置代理ip
//	 * @method
//	 */
//	private void setProxy(String proxyIP,int hostPort){
//		System.out.println("正在设置代理ip:"+proxyIP+":"+hostPort);
//		HttpHost proxy=new HttpHost(proxyIP, hostPort);
//		client.getParams().setParameter(ConnRouteParams.DEFAULT_PROXY, proxy);
//	}
	
//    /**
//     * 测试代理是否可用（其实和getHtml(String url, HttpHost proxyHost)的代码差不多，为了从功能上区别，暂时这样）
//     * @param httpHost 封装了代理的ip地址和端口
//     * @param url 用来测试的页面
//     * @return true 可用 false 不可用
//     */
//    public boolean isProxyUsable(String url) {
//    	String content = null;
//    	int statusCode = 500;
//    	
//    	for(int i=0;i<proxyIP.length;i++){
//        	setProxy(proxyIP[i], proxyPort[i]);
//            HttpGet httpGet = new HttpGet(url);
////            httpGet.getParams().setParameter(ConnRoutePNames.DEFAULT_PROXY, proxyHost);
//            try {
//                HttpResponse response = client.execute(httpGet);
//                
//                statusCode = response.getStatusLine().getStatusCode();
//                System.out.println(statusCode);
//                if(200 != statusCode) {
//                	if(i==proxyIP.length){
//                		System.out.println("这是最后一个ip。。。");
//                		break;
//                	}else {
//						continue;
//					}
//                    
//                }
//                HttpEntity entity = response.getEntity();
//                if(entity != null) {
//                	content = EntityUtils.toString(entity, "UTF-8");
//
//                    if(StringUtils.isNullOrEmpty(content)){
//                    	if(i==proxyIP.length){
//                    		System.out.println("这是最后一个ip。。。");
//                    		break;
//                    	}else {
//    						continue;
//    					}
//                    }
//                } else {
//                	if(i==proxyIP.length){
//                		System.out.println("这是最后一个ip。。。");
//                		break;
//                	}else {
//						continue;
//					}
//                }
//                
//            } catch (ClientProtocolException e) {
//                // TODO Auto-generated catch block
//                e.printStackTrace();
//                if(i==proxyIP.length){
//            		System.out.println("这是最后一个ip。。。");
//            		break;
//            	}else {
//					continue;
//				}
//            } catch (IOException e) {
//                // TODO Auto-generated catch block
//                e.printStackTrace();
//                if(i==proxyIP.length){
//            		System.out.println("这是最后一个ip。。。");
//            		break;
//            	}else {
//					continue;
//				}
//            }
//            return true;
//    	}
//        return true;
//    }
    
	public static void main(String[] args) throws ClientProtocolException, URISyntaxException, IOException{
		PageFetcher fetcher=new PageFetcher();
//		FetchedPage page=fetcher.getContentFromUrl("http://bj.ganji.com/fang1/1478318597x.htm");
//		FetchedPage page=fetcher.getHTMLbyProxy("http://bj.ganji.com/fang1/1469569753x.htm", "203.192.12.213", 80);
//		System.out.println(page.getUrl());
//		System.out.println(page.getStatusCode());
//		System.out.println(page.getContent());
//		fetcher.header();
//		fetcher.getHTML("http://bj.ganji.com/fang1/1469569753x.htm");
		int proxyIndex=1;
//		String url="http://click.ganji.com/click.php?url=YW15WjdsYlF0L3c0SjNSall2Q0RXLzZzN241bjRYNGhSanlSL3N6MzkwS3Avc2YvOFhzT09NQnU3UVRMR0dLY0w1dys5VDdCdi96QXhVM2pBOUF6cmM4TkxwWHJMNnVZR2EzZlJFc3VIc1N2Zmh2OUtMVnk4VTRVaDJ5MSs0UGgyZWd6MkVqMkVVamthVkh5Wjl5VU9KNzBXTFVtMWsxbDZlaWNJQUFjZzZHZDlKRDhDSUhrTHBVTXJyWWEzOUJXb3NEM3pTODVLK1dIRHZKZ2ExSjgyVTRwTFdlRTZCblRHTkRsMm1hKzZ6eGtKaUZzU2dlekZDWFk5eXhYM2U5NG5YVWJ0elBiTWFCYUNQdU1IeGdHS3pxOWlHcGVpeUNtYVRMUTVWc3RIL3Z6ZGFvRnp0THZFQT09";
		String url="http://bj.ganji.com/fang1/";
//		String url="http://bj.58.com/zufang/";
//		String url="http://jump.zhineng.58.com/clk?target=mv7V0A-b5HThmvqfpv--5HTduZGmR-wAXatQnDulXD6ZUhVLEy0q5iubpgPGujdbnjnYnHNdPiYLuHTvsHwhnjmVrj91naY1PjN1ujEzuHDkuyNh0vQfIjd_pgPYUARhIaubpgPYUHYQPjcOnjbOn1ckPHTzFMF-uWdCIZwkrBtfmhC8PH98mvqVsv6-XMNfFMPfIgFWuHYQFMuG0LRGujYkFMuG0LRG0jYznWc8rHc8nH9OsWndFhR8IZwk5HTh0A7zmyYqnHTdPWCkFh-1ug6YuyOb5yu6UZP-FMK_uWYVniuGUyRG5iudpyEqnWb3njczrjTLPjnQrjThmvQopyEquWEkrHTvP1NVPjT1nzYYmvmLsyDQuymVuANYmhNzPHNQrH0vFMKGujYOFhR8IA-b5HcQPWc3PWbYrHTzrjDdFhR8IZwk5HTh0ZFGmvNqPjbh0hRbpgcqpZwY0jCfsvFJsWN3shPfUiqCugGds1cQPWc3PWbYrHTzrjDdXaO1pZwVUaubpgPkULnqnBu1IyFGujYQnHNLrH03PzuW0hR6IARb5HDYPjEzPjnLn1ndP1N3PaukmgF6Uy-b5HDkPHmhuA-1mvDqnHTh0vR_uhP65HDkFhwG0vQf5HDh0vR_uhQf5HDh0HYksWT3PHcdn1DdPjnzrHmLnWELFMK6uvR8IyYqniuBpyEqPHThIv-_uAP60hEqnau60ZK-Uhwoug-LULFb5HTh0hRWmyQ_uhQfI1YkFh7BIjdhmyOMmv66U-tznjDdnjEQPgkQ";
//		String url="http://support.58.com/firewall/valid/3730619667.do?namespace=infodetailweb&url=http://bj.58.com/zufang/21632306237589x.shtml";
		FetchedPage page=new FetchedPage();
		page=fetcher.getHTMLbyProxy(url, Proxy.hostName[0], Proxy.port[0],"bj.ganji.com");
		System.out.println(page.getStatusCode()+page.getContent());
		System.out.println(page.getUrl());
		System.out.println(page.getContent().contains("请输入验证码"));
//		fetcher.getHTMLByCookie("http://bj.ganji.com/fang1/1469569753x.htm", Proxy.hostName[proxyIndex], Proxy.port[proxyIndex]);
		
	}
}
